package com.example.tts.service.impl;

import com.example.tts.service.ICrawlingService;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import java.util.ArrayList;

/**
 * @ClassName CSDNCrawlingService
 * @Description TODO
 * @Author 韩兴隆
 * @Date 2021/11/24 16:08
 **/
@Service
public class CSDNCrawlingService implements ICrawlingService {
    private WebClient webClient= new WebClient(BrowserVersion.CHROME);
    @Override
    public ArrayList<String> crawling(String URL){
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());

        HtmlPage page = null;
        try {
            page = webClient.getPage(URL);
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            webClient.close();
        }

        webClient.waitForBackgroundJavaScript(30000);
        String pageXml = page.asXml();
        Document document = Jsoup.parse(pageXml);
        Elements infoListEle = document.getElementById("pc_0_0").getElementsByTag("li");
        infoListEle.forEach(element -> {
            System.out.println(element.getElementsByClass("txt-box").first().getElementsByTag("h3").text());
            System.out.println(element.getElementsByClass("txt-box").first().getElementsByTag("a").attr("href"));
        });
        return new ArrayList<>();
    }
}
